#include "agent.h"

#include <QDebug>

#include "maze.h"

Agent::Agent( const Maze& m, double learningrate, double discountfactor, double explorationrate )
    : maze( m ),
      learningRate( learningrate ),
      discountFactor( discountfactor ),
      explorationRate( explorationrate )
{
    int w = maze.getWidth();
    int h = maze.getHeight();
    qTable.resize( w );
    for ( auto& r : qTable )
    {
        r.resize( h );
        for ( auto& c : r )
        {
            c.resize( 4 );
            c.fill( 0.0 );
            // todo: c.resize(4, 0.0);
        }
    }
    x = 1;
    y = 1;
}

void Agent::updateQValue( int currx, int curry, int action, int nextx, int nexty, double reward )
{
    double currQ    = qTable[currx][curry][action];
    double maxNextQ = *std::max_element( qTable[nextx][nexty].begin(), qTable[nextx][nexty].end() );
    double newQ     = currQ + learningRate * ( reward + discountFactor * maxNextQ - currQ );
    qDebug() << QString( "UPDATE point:(%1,%2)->(%3,%4), currQ=%5, maxnextQ=%6, newQ=%7, action=%8, "
                         "reward=%9" )
                    .arg( currx )
                    .arg( curry )
                    .arg( nextx )
                    .arg( nextx )
                    .arg( currQ )
                    .arg( maxNextQ )
                    .arg( newQ )
                    .arg( action )
                    .arg( reward );
    qTable[currx][curry][action] = newQ;
}

int Agent::chooseAction( int x, int y )
{
    if ( QRandomGenerator::global()->generateDouble() < explorationRate )
    {
        return QRandomGenerator::global()->bounded( 4 );
    }
    else
    {
        return std::distance( qTable[x][y].begin(), std::max_element( qTable[x][y].begin(), qTable[x][y].end() ) );
    }
}

void Agent::move( int action )
{
    int newx = x;
    int newy = y;
    switch ( action )
    {
        case 0:  // 上
            newy = y - 1;
            break;
        case 1:  // 下
            newy = y + 1;
            break;
        case 2:  // 左
            newx = x - 1;
            break;
        case 3:  // 右
            newx = x + 1;
        default:
            break;
    }

    if ( !maze.isWall( newx, newy ) )
    {
        x = newx;
        y = newy;
    }
}

int Agent::getX() const
{
    return x;
}

int Agent::getY() const
{
    return y;
}

void Agent::decayExplorationRate( double decayRate )
{
    explorationRate *= decayRate;
    if ( explorationRate < 0.01 )
    {
        explorationRate = 0.01;
    }
}
